API Location: https://aqicn.org/json-api/doc/
import json
import numpy as np
import pandas as pd
import requests as req
from scipy import stats
from datetime import datetime
This functions reads the CSV data from part 1, specifically retrieving the latitude and longitude of the stations collecting the air pollution data, and outputs the coordinates in the form of a dict by each district in Seoul.
def getStationCoordinates(airData):
# Get only unique values from each of the specified columns
district = airData['District'].unique()
lat = airData['Latitude'].unique()
long = airData['Longitude'].unique()
# Create dict containing each districts air station coordinates
coorDict = {}
for i in range(len(district)):
coorDict.update({district[i]: {'lat': lat[i], 'long': long[i]}})
return coorDict
Using the waqi API, this function retrieves air pollution information about each of the districts coordinates collected from the CSV data (gathered using the getStationCoordinates function above). Specifically getting the week forecast of PM10 and PM2.5 pollutants in the air.
def getStationData(coorDict, API_KEY_json_file):
# API URL and API Key
url = 'https://api.waqi.info/feed/'
with open(API_KEY_json_file) as key:
api_key = json.load(key)['API_KEY']
# Create a dict containing air pollution information about each district using their coordinates
stationData = {}
for district in coorDict.keys():
# Get the latitude and longitude of the district station
lat = coorDict[district]['lat']
long = coorDict[district]['long']
# Create the URL to get the information from the API
content = req.get(url + 'geo:' + str(lat) + ';' + str(long) + '/?token=' + api_key).json()
# Create the dict to store all the data gathered from the API
stationData.update({district: {'Coor': {'lat': content['data']['city']['geo'][0],
'long': content['data']['city']['geo'][1]},
'Data': {}}})
# Parse through the forecast data gathered from the API
# into the station data dict for later manipulation
for val in ['pm10', 'pm25']:
dates = []
avgs = []
for i in range(len(content['data']['forecast']['daily'][val])):
dates += [datetime.strptime(content['data']['forecast']['daily'][val][i]['day'],
'%Y-%m-%d').strftime('%m/%d/%Y')[1:]]
avgs += [content['data']['forecast']['daily'][val][i]['avg']]
stationData[district]['Data'][val] = {'Dates': dates, 'Avgs': avgs}
return stationData
This functions takes the data gathered from the API and formats it into a dataframe
def createAirData_Dataframe(stationData):
tempDfList = []
# Create a dataframe for each of the districts in the station data dict
for district in stationData.keys():
newDf = pd.DataFrame({
'Measurement_Date': stationData[district]['Data']['pm10']['Dates'],
'Country': ['Republic of Korea'] * len(stationData[district]['Data']['pm10']['Dates']),
'City': ['Seoul'] * len(stationData[district]['Data']['pm10']['Dates']),
'District': [district] * len(stationData[district]['Data']['pm10']['Dates']),
'Latitude': [stationData[district]['Coor']['lat']] * len(stationData[district]['Data']['pm10']['Dates']),
'Longitude': [stationData[district]['Coor']['long']] * len(stationData[district]['Data']['pm10']['Dates']),
'PM10': stationData[district]['Data']['pm10']['Avgs'],
'PM2.5': stationData[district]['Data']['pm25']['Avgs'],
})
tempDfList.append(newDf)
# Empty dataframe is created with the right column names
tempDf = pd.DataFrame(columns=['Measurement_Date', 'Country', 'City', 'District',
'Latitude', 'Longitude', 'PM10', 'PM2.5'])
# Loops through the temp dataframe list and appends them all into one dataframe
airData = tempDf.append([df for df in tempDfList])
return airData
This function applies a filter to the reformatted data to remove any outliers that might skew the data. The filter checks to make sure that the data in the data column(s) (PM10 and PM2.5) is within -3 and +3 standard deviations away from the mean for that column.
def filterData(data):
# The column(s) that hold numeric data
dataCols = ['PM10', 'PM2.5']
# Making sure that the data in data columns is numeric and not string
for col in dataCols:
data[col] = pd.to_numeric(data[col])
# This applies a filter to all the data columns of the dataframe:
# * For each column, it first computes the Z-score of each value
# in the column relative to the column mean and standard deviation.
# * If the score is not within -3 and +3 standard deviations away from the mean for that
# column, then the record is filtered out of the dataframe (thus removing the outliers)
filteredData = data[(np.abs(stats.zscore(data[dataCols])) < 3).all(axis=1)]
# This filter removes any data that is less than zero because
# the measurement of pollutants in the air cannot go below zero
filteredData = filteredData[(filteredData[dataCols] >= 0).all(axis=1)]
print('Total number of rows BEFORE data is removed: {:,}\n Total number of rows AFTER data is removed: {:,}\n'
'====================================================\n\t Total number of rows removed: {:>3,}'
.format(len(data.index), len(filteredData.index), len(data.index) - len(filteredData.index)))
return filteredData
This function calculates the AQI (Air Quality Index) value, typically calculated from PM2.5 and determines its risk level. The AQI formula, value ranges and risk levels were all taken from the EPA (Environmental Protection Agency) of the USA.
def calculate_AQI(airData):
airData = airData.reset_index(drop=True)
aqiValues = []
riskLevel = []
for i in range(len(airData)):
i_low = 0
i_high = 0
c_low = 0
c_high = 0
# PM2.5 AQI Value Calculation
if 0 <= airData['PM2.5'][i] <= 12:
c_low = 0
c_high = 12
i_low = 0
i_high = 50
elif 12.1 <= airData['PM2.5'][i] <= 35.4:
c_low = 12.1
c_high = 35.4
i_low = 51
i_high = 100
elif 35.5 <= airData['PM2.5'][i] <= 55.4:
c_low = 35.5
c_high = 55.4
i_low = 101
i_high = 150
elif 55.5 <= airData['PM2.5'][i] <= 150.4:
c_low = 55.5
c_high = 150.4
i_low = 151
i_high = 200
elif 150.5 <= airData['PM2.5'][i] <= 250.4:
c_low = 150.5
c_high = 250.4
i_low = 201
i_high = 300
elif 250.5 <= airData['PM2.5'][i] <= 350.4:
c_low = 250.5
c_high = 350.4
i_low = 301
i_high = 400
elif 350.5 <= airData['PM2.5'][i] <= 500.4:
c_low = 350.5
c_high = 500.4
i_low = 401
i_high = 500
# AQI Formula
aqiValues += [int(round(((i_high - i_low) / (c_high - c_low)) *
(airData['PM2.5'][i] - c_low) + i_low, 0))]
# Determine AQI Risk Level
if 0 <= aqiValues[i] <= 50:
riskLevel += ['Good']
elif 51 <= aqiValues[i] <= 100:
riskLevel += ['Moderate']
elif 101 <= aqiValues[i] <= 150:
riskLevel += ['Unhealthy for Sensitive Groups']
elif 151 <= aqiValues[i] <= 200:
riskLevel += ['Unhealthy']
elif 201 <= aqiValues[i] <= 300:
riskLevel += ['Very Unhealthy']
elif 301 <= aqiValues[i] <= 500:
riskLevel += ['Hazardous']
# Add the AQI values to the data frame
airData['AQI_(PM2.5)'] = aqiValues
airData['AQI_Risk_Level'] = riskLevel
return airData
airData = pd.read_csv('CSV-Air_Pollution_Data-(Reformed_and_AQI_Values).csv')
coorDict = getStationCoordinates(airData)
print('Total Number of District Stations: ' + str(len(coorDict.keys())) + '\n')
for district in coorDict.keys():
print('District: {:>15}, Coordinates: '.format(district) +
str(round(coorDict[district]['lat'], 8)) + ' (Latitude), ' +
str(round(coorDict[district]['long'], 8)) + ' (Longitude)')
# Replace WAQI_API_KEY.json with where your API Key is stored and make sure
# the key is specified as '"API_KEY": "<your key here>""' in the json file
stationData = getStationData(coorDict, 'WAQI_API_KEY.json')
# Just the first district in the dict to show the data's structure
stationData[list(stationData.keys())[0]]
airData = createAirData_Dataframe(stationData)
airData.head()
filteredData = filterData(airData)
airData = calculate_AQI(filteredData)
# Two new columns added: AQI_(PM2.5) and AQI_Risk_Level
print('New columns: ' + ', '.join(list(airData.keys())))
airData.head()
print('Final size of data: {:,} columns and {:,} rows'.format(airData.shape[1], airData.shape[0]))
airData.to_csv('Reformed_Data/API-Air_Pollution_Data-(Reformed_and_AQI_Values).csv', index=False)